#!/usr/bin/python
# -*- coding: UTF-8 -*-
import logging
import random

server_logger = logging.getLogger("server")

import time


import os,django
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "shareAPI2020TB.settings")# project_name 项目名称
django.setup()

import json
import jsonpath
import requests

from tbitem import models

'''
------本文件为副文件--------
get_3_tbContent.py ----- 把数据库中的淘宝id取出来，抓评论存储数据库

--------------------------
'''
def getOneID():
    #从数据库中取出未解析的链接
    querySet = models.BBSContent.objects.filter(status=1).all()
    return querySet
def getTwoID():
    #从数据库中取出未解析的链接
    querySet = models.DIYTitle.objects.filter(status=1).all()
    return querySet

def initlist(json_str):
    # 1初始化init
    items = []
    # 将json格式'str字符串'转化为python对象
    py_json = json.loads(json_str)
    # print(py_json)
    comments_list = py_json['comments']
    for comment in comments_list:
        # 使用jsonpath方式查找
        user = jsonpath.jsonpath(comment, '$..user')[0]
        # 用户头像
        user_face = 'http;' + user['avatar']
        # print(user_face)
        #用户名
        user_name=user['nick']
        # print(user_name)
        #评论内容
        user_text=comment['content']
        #评论日期
        user_date=comment['date']
        #手机信息
        info=jsonpath.jsonpath(comment, '$..sku')[0]
        #评论视频截图,有的没有视频video就是NULL值，所以加一个判断
        if 'VideoUrlideo' in comment:
            # 评论截图和视频
            videoimg = comment['video']['coverUrl']
            videoUrl = comment['video']['coverUrl']
        else:
            videoimg = None
            videoUrl = None

        #好评图片（comment['photos']是一个列表）
        imglist = []
        goodimglist = comment['photos']
        if goodimglist:
            for i in goodimglist:
                imglist.append('http;' +i['url'])

        item = {
            '评论内容': user_text,
            '好评图片': imglist,
        }
        print(item)
        dangerWord = ['垃圾','差评','破玩意','不要买','别买','差劲','他妈的','傻逼','坏的','此用户没有填写评价']  #过滤次
        for daner in dangerWord:
            if daner in item['评论内容']:
                print('发现不合适内容',daner)
                break
            # 3 append是加入到list
        items.append(item)

    return items


# 爬去京东的评论
def jdmain(id,page):
    HEADERS = [
        'User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
    ]

    url = f'https://club.jd.com/productpage/p-{id}-s-3-t-5-p-0.html?callback=fetchJSON_comment98vv37'
    headers = {
        'Accept': '*/*',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        'User-Agent': random.choice(HEADERS),

        'Connection': 'keep-alive',
        'Host': 'club.jd.com',
    }
    print(url)
    html = requests.get(url=url,headers=headers)
    print(html.text)
    print('----------------------------------------')

    content = html.text.split('({"')  #收到的str内容进行清洗处理，变成可用的json
    flag_two = content[1].split('}]}')  #收到的str内容进行清洗处理，变成可用的json
    json_flag = json.loads(f'''{{"{flag_two[0]}}}]}}''',strict=False)  #收到的str内容进行清洗处理，变成可用的json，处理好了

    return json_flag['comments']

def tbmain(id,page):
    url = f'https://rate.taobao.com/feedRateList.htm?auctionNumId={id}&userNumId=903511695&currentPageNum={page}&pageSize=20'
    headers = {
        'Accept': '*/*',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Referer': 'https://item.taobao.com/item.htm?spm=a230r.1.14.31.1a9df9eaZ2TtQ2&id=593014213362&ns=1&abbucket=4',

        'Cookie': 'miid=1034753504511266755;cna=dx1nFLT5mQcCAXWaR2QxwmBW;'

                  'thw=cn;t=325ff4e457909046382b22a33c83eadb;tracknick=acc%5Cu80E1%5Cu4F20;'

                  'lgc=acc%5Cu80E1%5Cu4F20;tg=0;v=0;'

                  'cookie2=128cea00303f12de470490c850cc00b1;_tb_token_=fdb4b335af83e;'

                  'unb=2701405241;sg=%E4%BC%A01a;_l_g_=Ug%3D%3D;skt=051d1e62c9ab8dc5;'

                  'cookie1=BxMH3NLGjomw6rJWdejLCTcGCFEkYwy%2BoQnBejFVBZI%3D;csg=93fdcb74;'

                  'uc3=vt3=F8dBy3jYcBxWmWeUeDY%3D;id2=UU8IPT1oZbApWA%3D%3D;nk2=AnIsGxSURQ%3D%3D;lg2=Vq8l%2BKCLz3%2F65A%3D%3D;'

                  'existShop=MTU1OTY1MDAxNw%3D%3D;_cc_=U%2BGCWk%2F7og%3D%3D;dnk=acc%5Cu80E1%5Cu4F20;_nk_=acc%5Cu80E1%5Cu4F20;'

                  'cookie17=UU8IPT1oZbApWA%3D%3D;mt=ci=6_1&np=;'

                  'enc=UFEUuOrXKTfwHDdKWP%2F9JacRMgPfTTQxKHxhkY8FExWK0%2FOKPqyryg048bj3QRjP%2Blwvl2%2BmU0N0pIBcrlD2PA%3D%3D;'
                  'hng=CN%7Czh-CN%7CCNY%7C156;'
                  'uc1=cookie15=VT5L2FSpMGV7TQ%3D%3D&cookie14=UoTZ7Yy7WSTGKg%3D%3D;'

                  '_m_h5_tk=3627342e3b09ae88a2c724b51c503d85_1559657695270;'
                  '_m_h5_tk_enc=c88d8e3f4ba68bc083f853e956656b6c;'
                  'x=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0%26__ll%3D-1%26_ato%3D0;'
                  'x5sec=7b22726174656d616e616765723b32223a22623034623263306164386231666234336634303330663463646430663064653743504c4f32656346454e364e6b736a586d664c7373774561444449334d4445304d4455794e4445374d513d3d227d;'
                  'whl=-1%260%260%261559652412550;'
                  'l=bB_eivGRvlAmYuqwBOfgiuI8a17TwIdfhsPzw4_G2ICP_bfDwBcPWZTPLZTkC3GVa6GXR3oyuHLBBW8iLyUCh;'
                  'isg=BGNjWHeLO7xJIPCeqwvJFNPV8qfN8Pf9ajjwFJXBBkM21IT2HSu26rnCyuT_9E-S',

        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',

        'Connection': 'keep-alive',
        'Host': 'rate.taobao.com',
    }
    print(f"爬取评论的连接是---{url}")
    html = requests.get(url=url, headers=headers)
    # print(html.text)
    print(html.url)
    json_str = html.text
    #清洗步骤（获取的字符串前后有括号）--去掉str前后括号
    json_str = json_str.strip().lstrip('(').rstrip(')')
    #str to JSON（dict）
    JSONlist = initlist(json_str)
    print("请求淘宝数据状态为")
    # print(JSONlist)
    return JSONlist


def step3():
    server_logger.error("step3-----Start")
    try:

        # tbid的取值模式：第一种bbs的采集入库表BBSContent，第二种，手动采集入库表DIYTitle
        run_modelQuery = models.RunModel.objects.first()
        if not run_modelQuery:
            logging.warning('请添加模式表，默认0是bbs发帖，1是调用手动表')
            raise Exception("请添加模式表，默认0是bbs发帖，1是调用手动表")

        # print(run_modelQuery.modelType)
        if run_modelQuery.modelType == 0:
            querySetList = getOneID() #第一种：数据库BBSContent取出还没爬的资源
        elif run_modelQuery.modelType == 1:
            querySetList = getTwoID()  # 第二种：数据库DIYTitle取出还没爬的资源
        else:
            querySetList = getOneID() #第一种：数据库BBSContent取出还没爬的资源

        for contentItem in querySetList:
            logging.warning('step3等待...')
            logging.warning('step3等待...')
            logging.warning('step3等待...')
            logging.warning('step3等待...')


            # 检测TbIndex里是否存在
            checkQuestId = models.TbIndex.objects.filter(tbid=contentItem.tbid).first()
            if not checkQuestId: #不存在就添加
                models.TbIndex.objects.create(tbid=contentItem.tbid).save()
                checkQuestId = models.TbIndex.objects.filter(tbid=contentItem.tbid).first()
            pages = 3  #到淘宝爬取评论
            for page in range(1,pages):
                JSONlist = tbmain(contentItem.tbid, page)  # 到淘宝爬取评论

                contentItem.status = 2  # 标记该bbsContent的内容已爬取
                contentItem.save()  # 标记该bbsContent的内容已爬取
                time.sleep(3)
                for item in JSONlist:
                    if item["评论内容"]:
                        models.TbComment.objects.update_or_create(tbComment_Index=checkQuestId, commentWord=item["评论内容"])
                    if len(item["好评图片"]) >= 1:
                        for img in item["好评图片"]:
                            # 插入内容到图片表
                            models.TbIMG.objects.update_or_create(tbimg_Index=checkQuestId, imgSrc=img.replace("http;", "http:"))
        server_logger.error("step3-----运行成功")
    except Exception as e:

        server_logger.error(f"step3-----step3出错----{e}")
if __name__ == '__main__':
    # step3()  #负责把bbs的内容遍历淘宝id，抓取淘宝评论到数据库

    tbmain(599739071372, 1)
